import scrapy
from wangyiSpider.items import WangyispiderItem


class JobSpider(scrapy.Spider):
    name = 'job'
    allowed_domains = ['163.com']
    start_urls = ['https://hr.163.com/position/list.do']

    def __init__(self):
        self.count = 0

    def parse(self, response):
        tr_list = response.xpath("//tbody/tr")
        # print(len(tr_list))
        for num, tr in enumerate(tr_list):
            if num % 2 == 0:
                item = WangyispiderItem()
                item["position"] = tr.xpath("./td/a/text()").extract_first()
                item["href"] = "https://hr.163.com" + tr.xpath("./td/a/@href").extract_first()
                item["department"] = tr.xpath("./td[2]/text()").extract_first()
                item["category"] = tr.xpath("./td[3]/text()").extract_first()
                item["type"] = tr.xpath("./td[4]/text()").extract_first()
                item["address"] = tr.xpath("./td[5]/text()").extract_first()
                item["num"] = tr.xpath("./td[6]/text()").extract_first().strip()
                item["date"] = tr.xpath("./td[7]/text()").extract_first()

                # 构建详情页面的请求
                yield scrapy.Request(
                    url=item['href'],
                    callback = self.parse_detail,
                    meta={"item": item}
                )

        # 模拟翻页
        next_page_url = response.xpath("//div[@class='m-page']/a[last()]/@href").extract_first()
        if next_page_url != "javascript:;":
            next_page_url = response.urljoin(next_page_url)
            yield scrapy.Request(
                url=next_page_url,
                callback=self.parse,
            )
            # self.count += 1
            # print(self.count)

    def parse_detail(self, response):
        # 将meta传递参数
        item = response.meta["item"]
        item['duty'] = "".join(response.xpath("//div[@class='detail-info']/div[1]/div/text()").extract())
        item["require"] = "".join(response.xpath("//div[@class='detail-info']/div[2]/div/text()").extract())
        print(item)


